{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "85f88996-cf08-4a53-b102-247e38229134",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-11-11T10:00:15.940182Z",
     "iopub.status.busy": "2024-11-11T10:00:15.939771Z",
     "iopub.status.idle": "2024-11-11T10:00:18.768654Z",
     "msg_id": "84e65420-a201-4ae3-9830-1d5cb73131e6",
     "shell.execute_reply": "2024-11-11T10:00:18.767898Z",
     "shell.execute_reply.started": "2024-11-11T10:00:15.940152Z"
    }
   },
   "outputs": [],
   "source": [
    "import pandas as pd \n",
    "import numpy as np \n",
    "import os \n",
    "import gc\n",
    "import warnings\n",
    "warnings.filterwarnings(\"ignore\")\n",
    "\n",
    "from sklearn.tree import DecisionTreeClassifier\n",
    "from sklearn.ensemble import RandomForestClassifier\n",
    "from sklearn.ensemble import GradientBoostingClassifier\n",
    "from sklearn.model_selection import train_test_split\n",
    "from sklearn.metrics import roc_curve\n",
    "from sklearn.model_selection import KFold, StratifiedKFold\n",
    "from sklearn.feature_selection import RFECV\n",
    "\n",
    "import lightgbm as lgb \n",
    "from lightgbm import log_evaluation, early_stopping\n",
    "import xgboost as xgb \n",
    "import copy \n",
    "\n",
    "from sklearn.ensemble import IsolationForest\n",
    "\n",
    "import matplotlib.pyplot as plt \n",
    "from matplotlib import rcParams\n",
    "rcParams[\"font.family\"] = \"SimHei\"\n",
    "\n",
    "import seaborn as sns\n",
    "\n",
    "from itertools import combinations \n",
    "import pickle\n",
    "# from bayes_opt import BayesianOptimization\n",
    "# import optuna\n",
    "from functools import partial\n",
    "\n",
    "import networkx as nx \n",
    "from itertools import combinations\n",
    "from functools import partial\n",
    "from sklearn.metrics import roc_auc_score\n",
    "from sklearn.preprocessing import LabelEncoder\n",
    "from sklearn.preprocessing import OneHotEncoder\n",
    "from sklearn.preprocessing import MinMaxScaler,StandardScaler\n",
    "from sklearn.model_selection import train_test_split\n",
    "from sklearn.linear_model import LogisticRegression\n",
    "from sklearn.linear_model import LogisticRegressionCV\n",
    "from sklearn.metrics import confusion_matrix,accuracy_score,classification_report,roc_auc_score,log_loss\n",
    "from sklearn.tree import DecisionTreeClassifier\n",
    "import sklearn.metrics as metrics\n",
    "from sklearn.metrics import classification_report\n",
    "from sklearn.metrics import roc_curve\n",
    "from sklearn.model_selection import KFold\n",
    "from sklearn.feature_selection import RFECV\n",
    "from sklearn.model_selection import StratifiedKFold\n",
    "import sklearn.ensemble as ensemble\n",
    "from sklearn.ensemble import RandomForestClassifier\n",
    "from sklearn.model_selection import cross_val_score\n",
    "from sklearn import svm\n",
    "from sklearn.feature_selection import SelectFromModel\n",
    "\n",
    "from sklearn.ensemble import RandomForestClassifier\n",
    "from sklearn.ensemble import GradientBoostingClassifier\n",
    "from sklearn.metrics import f1_score\n",
    "\n",
    "import xgboost as xgb\n",
    "from xgboost import XGBClassifier\n",
    "\n",
    "import lightgbm as lgb\n",
    "from lightgbm import LGBMClassifier\n",
    "from lightgbm import log_evaluation, early_stopping\n",
    "\n",
    "import catboost as cbt\n",
    "from catboost import CatBoostClassifier\n",
    "\n",
    "from scipy import stats,integrate\n",
    "from scipy.stats import ks_2samp\n",
    "#from scipy.stats import kssamp\n",
    "from scipy.stats import pearsonr\n",
    "from sklearn.model_selection import RandomizedSearchCV\n",
    "from scipy.stats import uniform\n",
    "from scipy.stats import kstest\n",
    "\n",
    "import toad\n",
    "\n",
    "pd.set_option('display.max_columns', 50)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "c0f923bd-1bec-46b2-8824-62214d871543",
   "metadata": {},
   "source": [
    "# 工具函数"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "aec48db7-13b0-42ab-bc57-4b98a0db7741",
   "metadata": {},
   "source": [
    "## 读取数据"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "e3b8f266-593c-413e-8267-9960e7d5eb96",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-11-11T10:00:19.744627Z",
     "iopub.status.busy": "2024-11-11T10:00:19.743644Z",
     "iopub.status.idle": "2024-11-11T10:00:19.755832Z",
     "msg_id": "19837b0b-481a-4fa5-9585-9df947bcd41c",
     "shell.execute_reply": "2024-11-11T10:00:19.754934Z",
     "shell.execute_reply.started": "2024-11-11T10:00:19.744593Z"
    }
   },
   "outputs": [],
   "source": [
    "def get_data(file_name, num_rows=None):\n",
    "    train_path = \"/home/mole/work/contest/train\"\n",
    "    test_path = \"/home/mole/work/contest/B\"\n",
    "    df_train = pd.read_csv(os.path.join(train_path, file_name + \"_T.csv\"), nrows=num_rows)\n",
    "    df_test = pd.read_csv(os.path.join(test_path, file_name + \"_B.csv\"), nrows=num_rows)\n",
    "    df_train[\"is_train\"] = 1\n",
    "    df_test[\"is_train\"] = 0\n",
    "    \n",
    "    df = pd.concat(objs=[df_train, df_test],axis=0)\n",
    "    df.rename(mapper = {'DATA_DAT': '数据日期', 'CUST_NO': '客户编号', 'OPTO': '经营期限至', 'OPFROM': '经营期限自', 'ENTSTATUS': '经营状态', 'REGCAP': '注册资本', 'ESDATE': '成立日期', 'FRNAME': '法定代表人/负责人/执行事务合伙人', 'ENTTYPE_CD': '企业（机构）类型编码', 'REGPROVIN_CD': '所在省份编码', 'INDS_CD': '国民经济行业代码', 'ALTDATE': '变更日期', 'ALTITEM': '变更事项', 'PERNAME': '人员姓名', 'POSITIONCODE': '职位代码', 'PERSONAMOUNT': '人员总数量', 'WEBTYPE': '网站（网店）类型', 'WEBSITNAME': '网站（网店）名称', 'DOMAIN': '网站（网店）地址', 'ANCHEDATE': '年报日期', 'ANCHEYEAR': '年报年份', 'EXECMONEY': '执行标的', 'REGDATECLEAN': '立案时间', 'COURTNAME': '执行法院', 'CASECODE': '案号', 'PUBLISHDATECLEAN': '发布时间', 'GISTID': '执行依据文号', 'PERFORMANCE': '被执行人履行情况', 'REGDATE': '立案时间', 'FINALDATE': '终本日期', 'UNPERFMONEY': '未履行金额', 'CONDATE': '出资日期', 'SUBCONAM': '认缴出资额（万元）', 'FUNDEDRATIO': '出资比例', 'INVTYPE': '股东类型', 'CONFORM': '出资方式', 'SH_CUST_NO': '股东客户编号', 'BTD_BEGINDATE': '所属日期起', 'BTD_ENDDATE': '所属日期止', 'BTD_COLLECTCODE': '征收项目代码', 'BTD_DECLARDATE': '申报日期', 'BTD_DECLARTERM': '申报期限', 'BTD_TOTALSALE': '全部销售收入', 'BTD_TAXABLESALE': '应税销售收入', 'BTD_TAXPAYABLE': '应纳税额', 'BTD_DEDUCTAMOUNT': '减免税额', 'TR_DAT': '交易日期', 'TR_CD': '交易代码', 'CHANL_CD': '渠道代码', 'ABS_INFO': '摘要信息', 'CPT_TYP_CD': '交易对手类型代码', 'ARG_ACCT_BAL': '合约账户余额', 'ACTG_DIRET_CD': '记账方向代码', 'TRS_CSH_IND': '现转标识', 'CSH_EX_IND': '钞汇标识', 'RMB_TR_AMT': '折人民币交易金额', 'CPT_INTL_FE_CUST_IND': '对手方行内客户标识', 'INT_BNK_TR_IND': '是否跨行交易', 'SAME_NAM_IND': '同名账户标识', 'CPT_CUST_NO': '交易对手客户编号'},\n",
    "              axis=1,\n",
    "              inplace=True\n",
    "             )\n",
    "    return df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "b92cbe60-099a-4da6-b9cd-4e6cf7e7fb2d",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-11-11T10:00:20.519370Z",
     "iopub.status.busy": "2024-11-11T10:00:20.518890Z",
     "iopub.status.idle": "2024-11-11T10:00:20.525588Z",
     "msg_id": "5e89fe9f-34a9-4749-95c6-f412ae87e2b7",
     "shell.execute_reply": "2024-11-11T10:00:20.524912Z",
     "shell.execute_reply.started": "2024-11-11T10:00:20.519339Z"
    }
   },
   "outputs": [],
   "source": [
    "def agg_statistics(df, group_cols, agg_functions, name_flag, p=False):\n",
    "    \"\"\"\n",
    "    分组聚合。\n",
    "    \"\"\"\n",
    "    ga = df.groupby(by=group_cols).agg(agg_functions)\n",
    "    ga.columns = [\"{}_{}_{}\".format(e[0], e[1], name_flag) for e in ga.columns.tolist()]\n",
    "    ga.reset_index(inplace=True)\n",
    "    \n",
    "    new_cols = [col for col in ga.columns.tolist() if col not in group_cols]\n",
    "    if p is True:\n",
    "        print(\"新聚合特征：\\n\", new_cols)\n",
    "    \n",
    "    return ga, new_cols"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "d970bd24-4399-446d-9b97-fd9b5f9db18d",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-11-11T10:00:21.860886Z",
     "iopub.status.busy": "2024-11-11T10:00:21.860411Z",
     "iopub.status.idle": "2024-11-11T10:00:21.869606Z",
     "msg_id": "3fba9342-3548-49b4-ac7f-b8eb9f5fb541",
     "shell.execute_reply": "2024-11-11T10:00:21.868855Z",
     "shell.execute_reply.started": "2024-11-11T10:00:21.860854Z"
    }
   },
   "outputs": [],
   "source": [
    "# 趋势差分特征衍生\n",
    "def get_kurt(series_x):\n",
    "    kurt = series_x.kurt()\n",
    "    return kurt\n",
    "    \n",
    "def trend_indicator(df, group_dim_1, group_dim_2, agg_functions, name_flag, offset=-1):\n",
    "    \"\"\"\n",
    "    group_dim_1：第一维度\n",
    "    group_dim_2：第二维度\n",
    "    \"\"\"\n",
    "    df = df.sort_values(by=group_dim_2, ascending=True)\n",
    "    ga = df.groupby(by=[group_dim_1, group_dim_2]).agg(agg_functions)\n",
    "    ga.columns = [\"{}_{}_{}\".format(e[0], name_flag, e[1]) for e in ga.columns.tolist()]\n",
    "    new_features = ga.columns.tolist()\n",
    "    ga.reset_index(inplace=True)\n",
    "\n",
    "    diff_new_features = []\n",
    "    for fea in new_features:\n",
    "        t = \"一阶差分_{}_{}\".format(fea, offset)\n",
    "        diff_new_features.append(t)\n",
    "        ga[t] = ga.groupby(by=group_dim_1)[fea].diff(offset) # -1\n",
    "\n",
    "    all_new_features = new_features + diff_new_features\n",
    "    agg_functions_tmp = {}\n",
    "    for fea in all_new_features:\n",
    "        agg_functions_tmp.update({fea:['last','mean','skew',get_kurt,'std','sum','max','min']})    \n",
    "\n",
    "    ga_new, _ = agg_statistics(df=ga, group_cols=[group_dim_1], agg_functions=agg_functions_tmp, name_flag=\"差分特征_{}\".format(name_flag))\n",
    "\n",
    "    return ga_new"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "cb2e163e-f4fc-45f3-a2c4-0d2aede47a94",
   "metadata": {},
   "source": [
    "# 模型训练"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "5b765a82-eb46-424c-9c43-bec755069d11",
   "metadata": {},
   "source": [
    "# 标签信息表"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "a3adb0a3-62d0-4672-9adc-fb30c177408b",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-11-11T10:00:31.854919Z",
     "iopub.status.busy": "2024-11-11T10:00:31.854434Z",
     "iopub.status.idle": "2024-11-11T10:00:31.930510Z",
     "msg_id": "d251d7b1-c51e-44e6-bce9-e58ca7835d4d",
     "shell.execute_reply": "2024-11-11T10:00:31.929803Z",
     "shell.execute_reply.started": "2024-11-11T10:00:31.854881Z"
    }
   },
   "outputs": [],
   "source": [
    "TARGET = get_data(\"XW_ENTINFO_TARGET\")"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "25b7f9c3-3e66-41f2-a206-9a693328dc6f",
   "metadata": {},
   "source": [
    "# 企业税务综合申报信息表"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "78d31ed0-28a0-4aca-94c8-5dd124704676",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-11-11T10:00:34.059741Z",
     "iopub.status.busy": "2024-11-11T10:00:34.059254Z",
     "iopub.status.idle": "2024-11-11T10:00:34.772913Z",
     "msg_id": "9cbb52e7-0eba-44fb-8093-25c13aea5f51",
     "shell.execute_reply": "2024-11-11T10:00:34.772119Z",
     "shell.execute_reply.started": "2024-11-11T10:00:34.059709Z"
    }
   },
   "outputs": [],
   "source": [
    "data = get_data(\"XW_ENTINFO_TAXDECLARE\").merge(TARGET[[\"客户编号\",\"FLAG\"]], how=\"left\", on=\"客户编号\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "2098af20-82ce-40b1-9596-12c5705b244c",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-11-11T10:00:34.900253Z",
     "iopub.status.busy": "2024-11-11T10:00:34.899871Z",
     "iopub.status.idle": "2024-11-11T10:00:36.008685Z",
     "msg_id": "adcfc31b-5f7d-4ab0-8631-8b2928d07b32",
     "shell.execute_reply": "2024-11-11T10:00:36.007902Z",
     "shell.execute_reply.started": "2024-11-11T10:00:34.900226Z"
    }
   },
   "outputs": [],
   "source": [
    "data[\"数据日期\"] = data[\"数据日期\"].astype(\"str\").astype(\"datetime64[ns]\")\n",
    "data[\"所属日期起\"] = data[\"所属日期起\"].astype(\"str\").astype(\"datetime64[ns]\")\n",
    "data[\"所属日期止\"] = data[\"所属日期止\"].astype(\"str\").astype(\"datetime64[ns]\")\n",
    "data[\"申报日期\"] = data[\"申报日期\"].astype(\"str\").astype(\"datetime64[ns]\")\n",
    "data[\"申报期限\"] = data[\"申报期限\"].astype(\"str\").astype(\"datetime64[ns]\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "317c9eac-d560-4077-b899-884ed508ee61",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-11-11T10:00:36.010414Z",
     "iopub.status.busy": "2024-11-11T10:00:36.010047Z",
     "iopub.status.idle": "2024-11-11T10:00:36.117922Z",
     "msg_id": "252793c4-6a39-4e5b-a857-d25a75b44b72",
     "shell.execute_reply": "2024-11-11T10:00:36.117229Z",
     "shell.execute_reply.started": "2024-11-11T10:00:36.010386Z"
    }
   },
   "outputs": [],
   "source": [
    "data[\"申报日期起_年份\"] = pd.DatetimeIndex(data['所属日期起']).year\n",
    "data[\"申报日期起_月份\"] = pd.DatetimeIndex(data['所属日期起']).month\n",
    "data[\"申报日期止_年份\"] = pd.DatetimeIndex(data['所属日期止']).year\n",
    "data[\"申报日期止_月份\"] = pd.DatetimeIndex(data['所属日期止']).month"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "78f4942e-08f0-49aa-bb0c-ddf3fa215525",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-11-11T10:00:36.912684Z",
     "iopub.status.busy": "2024-11-11T10:00:36.912260Z",
     "iopub.status.idle": "2024-11-11T10:00:37.060308Z",
     "msg_id": "3e9d2379-bdf8-4926-aa8f-48f65c2f2899",
     "shell.execute_reply": "2024-11-11T10:00:37.059586Z",
     "shell.execute_reply.started": "2024-11-11T10:00:36.912655Z"
    }
   },
   "outputs": [],
   "source": [
    "# 申报日期起年份_2000年条数很少，但1999和2000年坏率偏高\n",
    "# data[data[\"FLAG\"].notnull()][[\"申报日期起_年份\",\"FLAG\"]].groupby(by=\"申报日期起_年份\").agg([\"count\",\"mean\"])\n",
    "data[\"申报日期起_年份_是否2000年之前\"] = data[\"申报日期起_年份\"].apply(lambda x: 1 if x <= 2000 else 0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "40c4a030-c608-428f-9020-15a7a0ada8c4",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-11-11T10:00:39.330034Z",
     "iopub.status.busy": "2024-11-11T10:00:39.329546Z",
     "iopub.status.idle": "2024-11-11T10:00:39.608214Z",
     "msg_id": "a2827e75-352e-4bc5-92c3-638be683a215",
     "shell.execute_reply": "2024-11-11T10:00:39.607453Z",
     "shell.execute_reply.started": "2024-11-11T10:00:39.330004Z"
    }
   },
   "outputs": [],
   "source": [
    "data[\"报税间隔_天\"] = (data[\"所属日期止\"] - data[\"所属日期起\"]).dt.days\n",
    "data[\"报税缓冲天数\"] = (data[\"申报期限\"] - data[\"所属日期止\"]).dt.days\n",
    "data[\"税款是否预缴\"] = data[\"报税缓冲天数\"].apply(lambda x: 1 if x < 0 else 0)\n",
    "data[\"提前申报天数\"] = (data[\"申报期限\"] - data[\"申报日期\"]).dt.days\n",
    "data[\"是否在申报期限内申报\"] = data[\"提前申报天数\"].apply(lambda x: 1 if x >= 0 else 0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "493c3943-513a-4f6a-83d2-c6f053dc1784",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-11-11T10:00:40.108300Z",
     "iopub.status.busy": "2024-11-11T10:00:40.107695Z",
     "iopub.status.idle": "2024-11-11T10:00:40.398872Z",
     "msg_id": "2c28e0cd-39b4-4d42-ab16-80b8f0eb9e65",
     "shell.execute_reply": "2024-11-11T10:00:40.398091Z",
     "shell.execute_reply.started": "2024-11-11T10:00:40.108271Z"
    }
   },
   "outputs": [],
   "source": [
    "# 采样日期，在2002年7月、8月采样，标签为1年后，故应该以采样日期为基准加工指标\n",
    "tmp, _ = agg_statistics(data, group_cols=[\"客户编号\"], agg_functions={\"数据日期\":[\"max\"]}, name_flag=\"交易流水表\")\n",
    "data = data.merge(tmp, how=\"left\", on=\"客户编号\")\n",
    "\n",
    "# 加工不同税种最近一次报税日期 距离 采样日期的 时间长短\n",
    "# tmp, _ = agg_statistics(data, group_cols=[\"客户编号\"], agg_functions={\"申报日期\":[\"max\"]}, name_flag=\"交易流水表\")\n",
    "# data = data.merge(tmp, how=\"left\", on=\"客户编号\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "7e85702f-ff3d-480f-be4b-786397180988",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-11-11T10:00:42.063557Z",
     "iopub.status.busy": "2024-11-11T10:00:42.063070Z",
     "iopub.status.idle": "2024-11-11T10:00:42.118546Z",
     "msg_id": "9b4b127d-4366-4510-a3c9-a4de82bdddae",
     "shell.execute_reply": "2024-11-11T10:00:42.117772Z",
     "shell.execute_reply.started": "2024-11-11T10:00:42.063526Z"
    }
   },
   "outputs": [],
   "source": [
    "data[\"申报日期_距离_采样日期_天数\"] = (data[\"数据日期\"]- data[\"申报日期\"]).dt.days\n",
    "data[\"申报日期_距离_采样日期_月份数\"] = (data[\"数据日期\"]- data[\"申报日期\"]).dt.days // 30 + 1 # 1-20\n",
    "data[\"申报日期_距离_采样日期_季度数\"] = (data[\"数据日期\"]- data[\"申报日期\"]).dt.days // 120 + 1\n",
    "data[\"申报日期_距离_采样日期_年份数\"] = (data[\"数据日期\"]- data[\"申报日期\"]).dt.days // 365 + 1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "449531bf-ce35-4387-9d2a-c30469d97afe",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-11-11T10:00:42.780216Z",
     "iopub.status.busy": "2024-11-11T10:00:42.779781Z",
     "iopub.status.idle": "2024-11-11T10:00:43.851643Z",
     "msg_id": "0d8261c5-2123-4e5c-83ea-a3182a000f75",
     "shell.execute_reply": "2024-11-11T10:00:43.850831Z",
     "shell.execute_reply.started": "2024-11-11T10:00:42.780187Z"
    }
   },
   "outputs": [],
   "source": [
    "data[\"全部销售收入_是否小于0\"] = data[\"全部销售收入\"].apply(lambda x: 1 if x < 0 else 0) # 182笔\n",
    "data[\"全部销售收入_是否为0\"] = data[\"全部销售收入\"].apply(lambda x: 1 if x == 0 else 0) # \n",
    "data[\"应税销售收入_是否小于0\"] = data[\"应税销售收入\"].apply(lambda x: 1 if x < 0 else 0) # 4334\n",
    "data[\"应税销售收入_是否为0\"] = data[\"应税销售收入\"].apply(lambda x: 1 if x == 0 else 0)\n",
    "\n",
    "data[\"应纳税额_是否小于0\"] = data[\"应纳税额\"].apply(lambda x: 1 if x < 0 else 0) # 387\n",
    "data[\"应纳税额_是否为0\"] = data[\"应纳税额\"].apply(lambda x: 1 if x == 0 else 0)\n",
    "\n",
    "data[\"减免税额_是否小于0\"] = data[\"减免税额\"].apply(lambda x: 1 if x < 0 else 0) # 117\n",
    "data[\"减免税额_是否为0\"] = data[\"减免税额\"].apply(lambda x: 1 if x == 0 else 0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "4884e41d-6baf-4ed2-8c13-3947e3149ac1",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-11-11T10:00:43.853350Z",
     "iopub.status.busy": "2024-11-11T10:00:43.852987Z",
     "iopub.status.idle": "2024-11-11T10:00:51.565962Z",
     "msg_id": "f43a7af0-23e1-4bc6-b63c-cee29bf09a5f",
     "shell.execute_reply": "2024-11-11T10:00:51.565150Z",
     "shell.execute_reply.started": "2024-11-11T10:00:43.853323Z"
    }
   },
   "outputs": [],
   "source": [
    "data[\"企业税率\"] = data[[\"应税销售收入\",\"应纳税额\"]].apply(lambda x: round(x[1] / x[0],4)*100 if x[0]>0 and x[1] > 0 else np.nan, axis=1)\n",
    "data[\"减免税额_比上_应纳税额\"] = data[[\"减免税额\",\"应纳税额\"]].apply(lambda x: round(x[0] / x[1],4)*100 if x[0]>0 and x[1] > 0 else np.nan, axis=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "id": "ba867bd2-3f60-451a-af66-ab49b44f6382",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-11-11T10:00:51.567814Z",
     "iopub.status.busy": "2024-11-11T10:00:51.567448Z",
     "iopub.status.idle": "2024-11-11T10:00:51.575313Z",
     "msg_id": "d51aef4e-3505-4535-98aa-4f3cb54736ed",
     "shell.execute_reply": "2024-11-11T10:00:51.574593Z",
     "shell.execute_reply.started": "2024-11-11T10:00:51.567783Z"
    }
   },
   "outputs": [],
   "source": [
    "agg_functions = {\n",
    " \"申报日期起_年份_是否2000年之前\":[\"count\"],\n",
    " \"是否在申报期限内申报\":[\"sum\",\"mean\"],\n",
    " \"税款是否预缴\":[\"sum\",\"mean\"],\n",
    " \"报税间隔_天\":[\"min\",\"max\"],\n",
    " \"报税缓冲天数\":[\"min\",\"max\"],\n",
    " \n",
    " \"全部销售收入_是否小于0\":[\"sum\",\"mean\"],\n",
    " \"全部销售收入_是否为0\":[\"sum\",\"mean\"],\n",
    " \"应税销售收入_是否小于0\":[\"sum\",\"mean\"],\n",
    " \"应税销售收入_是否为0\":[\"sum\",\"mean\"],\n",
    " \"应纳税额_是否小于0\":[\"sum\",\"mean\"],\n",
    " \"应纳税额_是否为0\":[\"sum\",\"mean\"],\n",
    " \"减免税额_是否小于0\":[\"sum\",\"mean\"],\n",
    " \"减免税额_是否为0\":[\"sum\",\"mean\"],\n",
    " \"企业税率\":[\"mean\",\"min\",\"max\"],\n",
    " \"减免税额_比上_应纳税额\":[\"mean\",\"min\",\"max\"],\n",
    "\n",
    " \"全部销售收入\":[\"sum\",\"min\",\"max\",\"std\",\"mean\",\"count\"],\n",
    " \"应税销售收入\":[\"sum\",\"min\",\"max\",\"std\",\"mean\",\"count\"],\n",
    " \"应纳税额\":[\"sum\",\"min\",\"max\",\"std\",\"mean\",\"count\"],\n",
    " \"减免税额\":[\"sum\",\"min\",\"max\",\"std\",\"mean\",\"count\"],\n",
    "}\n",
    "agg_functions_diff =  {\n",
    "    \"全部销售收入\":[\"sum\",\"mean\"],\n",
    "    \"应税销售收入\":[\"sum\",\"mean\"],\n",
    "    \"应纳税额\":[\"sum\",\"mean\"],\n",
    "    \"减免税额\":[\"sum\",\"mean\"],\n",
    "}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "id": "e8f65712-54b9-460e-bc30-1758d94259c8",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-11-11T10:00:51.576577Z",
     "iopub.status.busy": "2024-11-11T10:00:51.576332Z",
     "iopub.status.idle": "2024-11-11T10:17:27.174238Z",
     "msg_id": "cf102b75-7f7b-43af-995d-b08d879c8e42",
     "shell.execute_reply": "2024-11-11T10:17:27.173435Z",
     "shell.execute_reply.started": "2024-11-11T10:00:51.576553Z"
    }
   },
   "outputs": [],
   "source": [
    "# 按照采样时间与申报时间的月份数，聚合\n",
    "df = copy.deepcopy(data)\n",
    "for is_train in [0, 1]:\n",
    "    data_agg = pd.DataFrame()\n",
    "    for mon in [24,1,3,6,12]:    \n",
    "        # 全部交易\n",
    "        temp1, _ = agg_statistics(df=df[(df[\"is_train\"]==is_train) & (df[\"申报日期_距离_采样日期_月份数\"]<=mon)],\n",
    "                                    group_cols=[\"客户编号\"],\n",
    "                                    agg_functions=agg_functions,\n",
    "                                    name_flag=\"最近{}个月申报_所有税种\".format(mon)\n",
    "                                    )\n",
    "\n",
    "        if data_agg.empty :\n",
    "            data_agg = copy.deepcopy(temp1)\n",
    "        else:\n",
    "            data_agg = data_agg.merge(temp1, how=\"left\", on=\"客户编号\")    \n",
    "\n",
    "        del temp1\n",
    "        gc.collect()\n",
    "        \n",
    "        for cod in [10101, 0, 10109, 30203, 30216, 10104, 10111]:\n",
    "            # 0\n",
    "            temp2, _ = agg_statistics(df=df[(df[\"is_train\"]==is_train) & (df[\"申报日期_距离_采样日期_月份数\"]<=mon) & (df[\"征收项目代码\"]==cod)], \n",
    "                                        group_cols=[\"客户编号\"],\n",
    "                                        agg_functions=agg_functions,\n",
    "                                        name_flag=\"最近{}个月申报_税种{}\".format(mon, cod)\n",
    "                                )\n",
    "            data_agg = data_agg.merge(temp2, how=\"left\", on=\"客户编号\")\n",
    "            \n",
    "            del temp2\n",
    "            gc.collect()\n",
    "\n",
    "    # 差分特征\n",
    "    temp1_diff = trend_indicator(df=df[(df[\"is_train\"]==is_train)],\n",
    "                                 group_dim_1=\"客户编号\",\n",
    "                                 group_dim_2=\"申报日期_距离_采样日期_月份数\",\n",
    "                                 agg_functions=agg_functions_diff, \n",
    "                                 name_flag=\"全部纳税流水_月度差分\")\n",
    "    data_agg = data_agg.merge(temp1_diff, how=\"left\", on=\"客户编号\")\n",
    "\n",
    "    temp2_diff = trend_indicator(df=df[(df[\"is_train\"]==is_train)],\n",
    "                                 group_dim_1=\"客户编号\",\n",
    "                                 group_dim_2=\"申报日期_距离_采样日期_季度数\",\n",
    "                                 agg_functions=agg_functions_diff, \n",
    "                                 name_flag=\"全部纳税流水_季度差分\")\n",
    "    data_agg = data_agg.merge(temp2_diff, how=\"left\", on=\"客户编号\")\n",
    "\n",
    "    temp3_diff = trend_indicator(df=df[(df[\"is_train\"]==is_train)],\n",
    "                                 group_dim_1=\"客户编号\",\n",
    "                                 group_dim_2=\"申报日期_距离_采样日期_年份数\",\n",
    "                                 agg_functions=agg_functions_diff, \n",
    "                                 name_flag=\"全部纳税流水_年度差分\")\n",
    "    data_agg = data_agg.merge(temp3_diff, how=\"left\", on=\"客户编号\")\n",
    "    \n",
    "    del temp1_diff,temp2_diff,temp3_diff\n",
    "    gc.collect()\n",
    "\n",
    "    for cod in [10101, 0, 10109, 30203, 30216, 10104, 10111]:\n",
    "        # 0\n",
    "        temp1_diff = trend_indicator(df=df[(df[\"is_train\"]==is_train) & (df[\"征收项目代码\"]==cod)],\n",
    "                                     group_dim_1=\"客户编号\",\n",
    "                                     group_dim_2=\"申报日期_距离_采样日期_月份数\",\n",
    "                                     agg_functions=agg_functions_diff, \n",
    "                                     name_flag=\"{}_纳税流水_月度差分\".format(cod))\n",
    "        data_agg = data_agg.merge(temp1_diff, how=\"left\", on=\"客户编号\")\n",
    "\n",
    "        temp2_diff = trend_indicator(df=df[(df[\"is_train\"]==is_train) & (df[\"征收项目代码\"]==cod)],\n",
    "                                     group_dim_1=\"客户编号\",\n",
    "                                     group_dim_2=\"申报日期_距离_采样日期_季度数\",\n",
    "                                     agg_functions=agg_functions_diff, \n",
    "                                     name_flag=\"{}_纳税流水_季度差分\".format(cod))\n",
    "        data_agg = data_agg.merge(temp2_diff, how=\"left\", on=\"客户编号\")\n",
    "\n",
    "        temp3_diff = trend_indicator(df=df[(df[\"is_train\"]==is_train) & (df[\"征收项目代码\"]==cod)],\n",
    "                                     group_dim_1=\"客户编号\",\n",
    "                                     group_dim_2=\"申报日期_距离_采样日期_年份数\",\n",
    "                                     agg_functions=agg_functions_diff, \n",
    "                                     name_flag=\"{}_纳税流水_年度差分\".format(cod))\n",
    "        data_agg = data_agg.merge(temp3_diff, how=\"left\", on=\"客户编号\")\n",
    "        \n",
    "\n",
    "        \n",
    "        del temp2_diff\n",
    "        gc.collect()\n",
    "    \n",
    "    if is_train == 0:\n",
    "        test_agg = copy.deepcopy(data_agg)\n",
    "    else:\n",
    "        train_agg = copy.deepcopy(data_agg)\n",
    "\n",
    "df_final = train_agg.append(test_agg)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "c29ed38f-bf3f-400b-95f2-f07c026dd512",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-11-11T10:18:17.831164Z",
     "iopub.status.busy": "2024-11-11T10:18:17.830665Z",
     "iopub.status.idle": "2024-11-11T10:18:18.752287Z",
     "msg_id": "b0b52099-ec71-4fa0-86c3-1d4ce4bf4781",
     "shell.execute_reply": "2024-11-11T10:18:18.751412Z",
     "shell.execute_reply.started": "2024-11-11T10:18:17.831132Z"
    }
   },
   "outputs": [],
   "source": [
    "# o_df_final = pd.read_pickle(\"/home/mole/work/xukunzhou/B榜/20241104/data/企业税务综合申报信息表_暴力衍生_1028.pkl\")\n",
    "\n",
    "# df_final.equals(o_df_final)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "37847bb6-816a-41b1-9800-257721b2674d",
   "metadata": {},
   "outputs": [],
   "source": [
    "# df_final.to_pickle(\"/home/mole/work/xukunzhou/B榜/20241104/data/企业税务综合申报信息表_暴力衍生_1028.pkl\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "3a7de7ca-22fe-464a-95d8-3c88f7d60f62",
   "metadata": {},
   "outputs": [],
   "source": [
    "df_final.to_pickle(\"../data/B_企业税务综合申报信息表_暴力衍生_1028.pkl\")"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.18"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}
